{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pima Indians Diabetes Data Set模型训练\n",
    "数据说明：Pima Indians Diabetes Data Set（皮马印第安人糖尿病数据集） 根据现有的医疗信息预测5年内皮马印第安人糖尿病发作的概率。  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.import工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings \n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "      <th>Pregnancies_tfidf</th>\n",
       "      <th>Glucose_tfidf</th>\n",
       "      <th>BloodPressure_tfidf</th>\n",
       "      <th>SkinThickness_tfidf</th>\n",
       "      <th>Insulin_tfidf</th>\n",
       "      <th>BMI_tfidf</th>\n",
       "      <th>DiabetesPedigreeFunction_tfidf</th>\n",
       "      <th>Age_tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "      <td>0.521271</td>\n",
       "      <td>0.800996</td>\n",
       "      <td>-0.367192</td>\n",
       "      <td>0.308084</td>\n",
       "      <td>-0.386666</td>\n",
       "      <td>-0.187761</td>\n",
       "      <td>0.315661</td>\n",
       "      <td>1.107942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.789470</td>\n",
       "      <td>-0.895213</td>\n",
       "      <td>0.201283</td>\n",
       "      <td>0.465320</td>\n",
       "      <td>0.684506</td>\n",
       "      <td>-0.180681</td>\n",
       "      <td>-0.186748</td>\n",
       "      <td>0.259045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "      <td>0.907893</td>\n",
       "      <td>1.690672</td>\n",
       "      <td>-0.998930</td>\n",
       "      <td>-0.445372</td>\n",
       "      <td>-0.690700</td>\n",
       "      <td>-1.394245</td>\n",
       "      <td>0.299983</td>\n",
       "      <td>-0.502676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.740472</td>\n",
       "      <td>-0.021457</td>\n",
       "      <td>0.776463</td>\n",
       "      <td>0.140992</td>\n",
       "      <td>-0.044228</td>\n",
       "      <td>0.506876</td>\n",
       "      <td>-0.745966</td>\n",
       "      <td>-0.446429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.139933</td>\n",
       "      <td>0.051123</td>\n",
       "      <td>-2.041679</td>\n",
       "      <td>0.146750</td>\n",
       "      <td>0.728256</td>\n",
       "      <td>0.546757</td>\n",
       "      <td>4.568139</td>\n",
       "      <td>-0.363719</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies   Glucose  BloodPressure  SkinThickness   Insulin       BMI  \\\n",
       "0     0.639947  0.866045      -0.031990       0.670643 -0.181541  0.166619   \n",
       "1    -0.844885 -1.205066      -0.528319      -0.012301 -0.181541 -0.852200   \n",
       "2     1.233880  2.016662      -0.693761      -0.012301 -0.181541 -1.332500   \n",
       "3    -0.844885 -1.073567      -0.528319      -0.695245 -0.540642 -0.633881   \n",
       "4    -1.141852  0.504422      -2.679076       0.670643  0.316566  1.549303   \n",
       "\n",
       "   DiabetesPedigreeFunction       Age  Outcome  Pregnancies_tfidf  \\\n",
       "0                  0.468492  1.425995        1           0.521271   \n",
       "1                 -0.365061 -0.190672        0          -0.789470   \n",
       "2                  0.604397 -0.105584        1           0.907893   \n",
       "3                 -0.920763 -1.041549        0          -0.740472   \n",
       "4                  5.484909 -0.020496        1          -1.139933   \n",
       "\n",
       "   Glucose_tfidf  BloodPressure_tfidf  SkinThickness_tfidf  Insulin_tfidf  \\\n",
       "0       0.800996            -0.367192             0.308084      -0.386666   \n",
       "1      -0.895213             0.201283             0.465320       0.684506   \n",
       "2       1.690672            -0.998930            -0.445372      -0.690700   \n",
       "3      -0.021457             0.776463             0.140992      -0.044228   \n",
       "4       0.051123            -2.041679             0.146750       0.728256   \n",
       "\n",
       "   BMI_tfidf  DiabetesPedigreeFunction_tfidf  Age_tfidf  \n",
       "0  -0.187761                        0.315661   1.107942  \n",
       "1  -0.180681                       -0.186748   0.259045  \n",
       "2  -1.394245                        0.299983  -0.502676  \n",
       "3   0.506876                       -0.745966  -0.446429  \n",
       "4   0.546757                        4.568139  -0.363719  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "dpath = 'D:/Jupyter Notebook/CSDN/The fifth week/'\n",
    "\n",
    "train1 = pd.read_csv(dpath +'FE_pima-indians-diabetes_org.csv')\n",
    "train2 = pd.read_csv(dpath +'FE_pima-indians-diabetes_log.csv')\n",
    "train3 = pd.read_csv(dpath +'FE_pima-indians-diabetes_tfidf.csv')\n",
    "#去掉多余的id\n",
    "train2 = train2.drop(['Outcome'], axis=1)\n",
    "train3 = train3.drop(['Outcome'], axis=1)\n",
    "train =  pd.concat([train1,train3], axis = 1, ignore_index=False)\n",
    "del train1,train2,train3\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Outcome']   \n",
    "X_train = train.drop(['Outcome'], axis=1)\n",
    "\n",
    "# 保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "# sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "# 查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "# 可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 默认参数的Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.49452634 0.52692738 0.45638625 0.4135294  0.48733779]\n",
      "cv logloss is： 0.4757414334285842\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#数据集比较大，采用3折交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is：',-loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "原始特征：0.47615662604739944<br/>\n",
    "log特征：0.47095211705658413<br/>\n",
    "tfidf特征：0.5757052287749229<br/>\n",
    "\n",
    "原始特征 + log特征：0.4599898998230634<br/>\n",
    "原始特征 + tfidf特征：0.4757414334285842<br/>\n",
    "log特征 + tfidf特征：0.45815192501490937<br/>\n",
    "\n",
    "原始特征 + log特征 + tfidf特征： 0.4618827287680177<br/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L1/L2） <br/>\n",
    "目标函数为：J =  C* sum(logloss(f(xi), yi)) + penalty  <br/>\n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同： <br/>\n",
    "1.设置参数搜索范围 <br/>\n",
    "2.生成GridSearchCV的实例（参数） <br/>\n",
    "3.调用GridSearchCV的fit方法 <br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='auto',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='lbfgs',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "                         'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# 需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters, cv=5, scoring='neg_log_loss', return_train_score=True)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47355076828181675\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.原始特征：0.4759758458327414{'C': 0.1, 'penalty': 'l2'}<br/>               对应无正则化：0.47615662604739944<br/>\n",
    "2.log特征：0.4709121521032231{'C': 0.1, 'penalty': 'l2'} <br/>              对应无正则化：0.47095211705658413<br/>\n",
    "3.tfidf特征：0.5757052287749229{'C': 1, 'penalty': 'l2'} <br/>              对应无正则化：0.5757052287749229<br/>\n",
    "4.原始特征 + log特征：0.4599898998230634{'C': 1, 'penalty': 'l2'}  <br/>       对应无正则化：0.4599898998230634<br/>\n",
    "5.原始特征 + tfidf特征：0.47355076828181675{'C': 0.1, 'penalty': 'l2'}  <br/>    对应无正则化：0.4757414334285842<br/>\n",
    "6.log特征 + tfidf特征：0.45815192501490937{'C': 1, 'penalty': 'l2'}<br/>        对应无正则化：0.45815192501490937<br/>\n",
    "7.原始特征 + log特征 + tfidf特征：0.4618827287680177{'C': 1, 'penalty': 'l2'} <br/> 对应无正则化：0.4618827287680177<br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXwV1fn48c+TfYVECFvCEhAkbIEaWdSigBUEVBaBREHFWrQFC3a1aq1L3Wu/WrU/VBQ3JCCbiAjUFbXVCggKhq2AkLCKCSGQPc/vj3sTQ7jATXKHm+V5t/eVmbnnzDwDOE/OmZlzRFUxxhhjqgrwdwDGGGPqJksQxhhjPLIEYYwxxiNLEMYYYzyyBGGMMcajIH8H4EvNmzfXDh06+DsMY4ypN9auXfu9qsZ5+q5BJYgOHTqwZs0af4dhjDH1hoh8d6rvrIvJGGOMR5YgjDHGeGQJwhhjjEcN6h6EMabxKi4uJjMzk4KCAn+HUieFhYWRkJBAcHCw13UsQRhjGoTMzEyio6Pp0KEDIuLvcOoUVeXw4cNkZmaSmJjodT3rYjLGNAgFBQU0a9bMkoMHIkKzZs2q3bqyBGGMaTAsOZxaTf5sLEEYYxqtCc/9hwnP/cffYdRZliCAySsmM3nFZH+HYYyp56KioiqWhw0bRkxMDCNHjvRYdurUqfTu3Ztu3boRHh5O79696d27NwsWLKjWMdetW8eKFStqFfep2E1qY4xxwO9//3uOHz/Oc8895/H7Z599FoBdu3YxcuRI1q9fX6PjrFu3jo0bNzJs2LAax3oq1oIwxhgHDBkyhOjo6BrV3bZtG0OHDuX8889n4MCBbN26FYD09HR69OhBcnIygwYNIj8/n/vvv585c+bUqPVxJtaCMMY0OPe9vYlv9+aesdy3+1xlvLkP0a1NE/5yZfdax+aNKVOmMGvWLDp16sRnn33GtGnTWLVqFffddx8fffQRLVu2JCcnh/DwcO655x42btzIk08+6fM4LEEYY0wdkpOTw+eff87YsWMrtpWUlABw0UUXcf311zNu3DjGjBnjeCyWIIwxDY63v+mXtxzm3TLAyXCqRVVp3ry5x3sSL7zwAl988QXLli0jOTmZr7/+2tFY7B4EUFBSQFFpkb/DMMYYYmNjad26NYsXLwagrKyMDRs2ALBjxw769+/PAw88QGxsLFlZWURHR3P06FFHYmn0CSKvKI8tBzYS+fkmf4dijGlAfvrTnzJu3Djef/99EhISWLlypdd109PTmTlzJsnJyXTv3p1ly5YBcPvtt9OzZ0969uzJZZddRo8ePRg8eDAbNmygT58+dpPa16JCouizO5A1HUo5UniEpqFN/R2SMaaeysvLq1j+5JNPvKrToUMHNm7ceMK2jh07ekwoS5cuPWlbXFycYxOlOdqCEJFhIrJFRLaLyB0evr9URI6IyHr3555K390uIptEZKOIzBWRMCdi1KIirv4C2h1UFm5b6MQhjDF11LxbBtSp+w91jWMJQkQCgWeBK4BuQJqIdPNQ9BNV7e3+3O+uGw/8GkhR1R5AIJDqRJyqSsfdhVz7cRlzN8+lpKzEicMYY0y942QLoi+wXVV3qGoRkA5cXY36QUC4iAQBEcBeB2IkIDSUY9HBdN+lFO7fx/u733fiMMYYU+84mSDigT2V1jPd26oaICIbRORdEekOoKpZwN+A3cA+4IiqrvJ0EBGZIiJrRGTNoUOHahRoXtMQAoAxGdG8/u3rNdqHMcY0NE4mCE9jy2qV9XVAe1VNBp4GlgCISCyu1kYi0AaIFJGJng6iqs+raoqqpsTFxdUo0JLgAPLDAxn0VSlfH/iKjd9vPHMlY4xp4JxMEJlA20rrCVTpJlLVXFXNcy8vB4JFpDlwGbBTVQ+pajGwCLjQqUDTb+vOu9d2IuSHo1y0M5TXM6wVYUyjMHuE62M8cjJBfAl0FpFEEQnBdZP5hGe0RKSVuGexEJG+7ngO4+pa6i8iEe7vhwAZTgU6e9hs7rx9IUGtWpG6qSkrd67k4PGDTh3OGNNAne3hvhcvXszjjz9e67hPxbH3IFS1RESmAStxPYX0kqpuEpFb3d/PBK4BfikiJUA+kKqqCnwhIgtwdUGVAF8BzzsVK4AEBREzfhwl/3iauIuDmLdlHrf1uc3JQxpjGjBfDfddUlJCUJDnS/Xo0aN9E+wpOPoehKouV9UuqtpJVR90b5vpTg6o6jOq2l1Vk1W1v6r+u1Ldv6hqV1XtoaqTVLXQyVgBYq65BoKCmLy9DW9ueZPCUscPaYxpoGoz3PfFF1/MXXfdxcCBA3nmmWd466236NevH3369OHyyy/n4EFXD8esWbOYMWMGABMnTmT69OlceOGFdOzYsWKojtpo9G9SVxbcogXRl11G8mefkNengOU7ljO6s7MZ2hjjgHfvgP3fnLncfvdgd97ch2jVE654pHZxVUNubi6rV68GIDs7m6uuugoRYebMmTzxxBM8+uijJ9U5ePAgn332Gd988w3jx4+vdQvDEkQVsampHF2xgjG72/BaxmuMOneUTYRujDnrUlN/fDd49+7djB8/nv3791NYWEiXLl081hk1ynW96tWrF1lZWbWOwRJEFRH9+hLSsSPD1xcxr9M2/rv/v/Rr3c/fYRljqsPb3/TLWw6T33EulhqKjIysWJ46dSp33nknw4cP57333uORRzyfX2hoaMWy63Zu7TT60VyrEhFiU1MJ35pJ7x+i7ZFXY4zfHTlyhPj4eFSVV1555awd1xKEB01HXY2Eh3PD1lZ8vOdjdufu9ndIxph6pjbDfVd17733Mnr0aC655BJatmzpwyhPT3zRDKkrUlJS1FfD3u7785/Jefttbp4KVyanckffkwajNcbUIRkZGSQlJVWvUh3uYnKCpz8jEVmrqimeylsL4hRiUlOhoJApmV1Ysn0JeUV5Z65kjKlfJr/TaJJDTViCOIXw7t0J69WLCz7/gWNFeSzeXvtnio0xpj6xBHEasWlpyHdZjMrtzJyMOZSWlfo7JGOMOWssQZxGkyuGEdC0KaO+DiMrL4uPMz/2d0jGGHPWWII4jYCwMGJGjybi842cV9bCHnk1xjQqliDOIDZ1ApSUMGVXR77c/yWbf9js75CMMT4yecVkJq+Y7O8w6ixLEGcQ0qEDkRdeSPsPtxAZEMacjDn+DskYU0eVD/e9fv16BgwYQPfu3enVqxfz5s07qawvhvsGWLduHStWrPBJ/FXZUBteiL02jWPTbmNK7kCe2fEOM34yg2bhzfwdljGmjoqIiODVV1+lc+fO7N27l/PPP5+hQ4cSExNTUcbb4b7PZN26dWzcuJFhw4b5JPbKrAXhhahLLyWoVSsu+m8exWXFzN86398hGWPqsC5dutC5c2cA2rRpQ4sWLTh06JDX9bdt28bQoUM5//zzGThwIFu3bgUgPT2dHj16kJyczKBBg8jPz+f+++9nzpw5NWp9nIm1ILwgQUHEjLuG759+hhFD+zJ/y3x+3uPnhASG+Ds0Y4wHj/73Ua/uF5aX8eY+RNdzuvLHvn+sdiz//e9/KSoqolOnTl7XmTJlCrNmzaJTp0589tlnTJs2jVWrVnHffffx0Ucf0bJlS3JycggPD+eee+5h48aNPPnkk9WO7UysBeGlmGvGQVAQEzJi+D7/e1buqvm4KsaYxmHfvn1MmjSJ2bNnExDg3eU2JyeHzz//nLFjx9K7d2+mTp3K3r17Abjooou4/vrrmTVrFmVlZU6GDjjcghCRYcBTuKYcnaWqj1T5/lLgLWCne9MiVb3f/V0MMAvoAShwk6r+x8l4Tye4ZQuihwwhYNXndOnVgde+fY2RHUfaXBHG1EHe/qZf3nKYPWy2z2PIzc1lxIgR/PWvf6V///5e11NVmjdv7vGexAsvvMAXX3zBsmXLSE5O5uuvv/ZlyCdxrAUhIoHAs8AVQDcgTUS6eSj6iar2dn/ur7T9KWCFqnYFkoEMp2L1VmxaKmVHjnDL9z3J+CGDrw5+5e+QjDF1UFFREaNHj+b6669n3Lhx1aobGxtL69atK6YMLSsrY8OGDQDs2LGD/v3788ADDxAbG0tWVhbR0dEcPXrU5+cAznYx9QW2q+oOVS0C0oGrvakoIk2AgcCLAKpapKo5jkXqpYh+/QhJTKTTh9toEtLEXpwzxng0f/58Vq9ezcsvv1zx+Gp1nlJKT09n5syZJCcn0717d5YtWwbA7bffTs+ePenZsyeXXXYZPXr0YPDgwWzYsIE+ffrUq5vU8cCeSuuZgKep2QaIyAZgL/A7Vd0EdAQOAbNFJBlYC0xX1WNVK4vIFGAKQLt27Xx7Bicfi9i0VA489DA3jR/DP3YvY2/eXtpEtXH0uMaY+iEvzzXq88SJE5k4caJXdTp06MDGjRtP2NaxY0eP80csXbr0pG1xcXH4apqDqpxsQXjqnK86+cQ6oL2qJgNPA0vc24OAnwD/T1X7AMcAjxMyqOrzqpqiqilxcXG+ifw0mo4ahYSFcemaQgRh7ua5jh/TGOOM2cNmO3L/oaFwMkFkAm0rrSfgaiVUUNVcVc1zLy8HgkWkubtupqp+4S66AFfC8LvAJk1oMnIExSs/YHiLS1m4dSHHi4/7OyxjjPE5JxPEl0BnEUkUkRAgFTihfSQircT9GJCI9HXHc1hV9wN7ROQ8d9EhwLcOxlotsalpaH4+qTtbcbT4KEv/d3Kzzxhj6jvHEoSqlgDTgJW4nkCar6qbRORWEbnVXewaYKP7HsQ/gFT9cQ7U24A5IvI10Bt4yKlYqyu8R3fCevYkctkn9GzWgzkZcyhT559JNsaYs8nRF+VUdbmqdlHVTqr6oHvbTFWd6V5+RlW7q2qyqvZX1X9XqrvefW+hl6qOUtVsJ2Otrti0NIr+t4ObSwawK3cXn2Z96u+QjDHGp+xN6hpqMvwKApo2pfOH/6NFeAsb5dWYeui7Sdfz3aTr/R1GnWUJooYCwsKIGTWKvPc/YFLLkfx777/Znr3d32EZY/zobA/3vXjxYh5//HGfxV+VDdZXCzGpE/jhlVcY8rXyTGwoczbP4S8D/uLvsIwxfubL4b5LSkoICvJ8qR49erTvg6/EWhC1EJqYSOSFAyhY+DZXth/Osv8tI6fA7y98G2P8rLbDfV988cXcddddDBw4kGeeeYa33nqLfv360adPHy6//HIOHjwIwKxZs5gxYwbgejlv+vTpXHjhhXTs2LFiqI7asBZELcWkppL16+mk/tCZBaWLWbBtATf3vNnfYRnTqO1/6CEKM8483HfBZlcZb+5DhCZ1pdWdd1Y7lpoM9w2uwf5Wr14NQHZ2NldddRUiwsyZM3niiSd49NFHT6pz8OBBPvvsM7755hvGjx9f6xaGJYhaih48mKAWLQh9ezX9x/dn7ua53ND9BoIDgv0dmjHGz8qH+37llVe8Hu67XGpqasXy7t27GT9+PPv376ewsJAuXbp4rDNq1ChEhF69epGVlVWr2MESRK1JUBAx48fz/TPPcMPN9/PLfffz/nfvMyzR99P/GWO84+1v+uUth/avverzGGo63He5yMjIiuWpU6dy5513Mnz4cN577z0eeeQRj3VCQ0Mrln98pazm7B6ED8SMuwYCAzl39U7aRbfjtYzX/B2SMcaPajPctydHjhwhPj4eVeWVV17xQYTesQThA8EtWxI9ZAi5CxdxXcdxfH3oa74+5OxEHsaYuqu2w31Xde+99zJ69GguueQSWrZs6cNIT0980QypK1JSUtSpYW/P5Nh//sPuyTdxzoP3MqrwSX6a8FMeG/iYX2IxpjHKyMggKSmpWnWc7GKqizz9GYnIWlVN8VTeWhA+EtG/PyEdOpA/fzFjOo/hX7v+xf5j+/0dljHmNNq/9mqjSQ41YQnCR8onE8rfsIHxcgFllDFvy8lvTxpjTH1hCcKHyicTCnnrAwa1HcSCrQvIL8n3d1jGNBoNqcvc12ryZ2MJwocCmzalyYjhHFm2jIltx5BTmMM7O97xd1jGNAphYWEcPnzYkoQHqsrhw4cJCwurVj17D8LHYlPTOLJwER0/303Xc7oyJ2MOYzuPxT0vkjHGIQkJCWRmZlZrSIvGJCwsjISEhGrVsQThY+E9exDWowfZc9OZ+OTN3P3vP/P5vs8Z0GaAv0MzpkELDg4mMTHR32E0KNbF5ADXZEL/49LDLTgn7Bxez3jd3yEZY0y1OZogRGSYiGwRke0icoeH7y8VkSMist79uafK94Ei8pWILHMyTl9rMvwKApo0IW/+AiacN4HVmav5Lvc7f4dljDHV4liCEJFA4FngCqAbkCYi3TwU/URVe7s/91f5bjqu+azrlYDwcGJGjyJ31b+4ptkQggOCbcY5Y0y942QLoi+wXVV3qGoRkA5c7W1lEUkARgCzHIrPUTETUqGkhIB3PuSKxCtYsn0JuUW5/g7LGGO85mSCiAf2VFrPdG+raoCIbBCRd0Wke6XtTwJ/AMpOdxARmSIia0RkTV16eiG0YyIRA/qTPW8+E7ukkV+Sz+JttZ/AwxhjzhYnE4Sn5zqrPqC8DmivqsnA08ASABEZCRxU1bVnOoiqPq+qKaqaEhcXV9uYfSo2NY2SfftI2HiQ81uezxsZb1BSVuLvsIwxxitOJohMoG2l9QRgb+UCqpqrqnnu5eVAsIg0By4CrhKRXbi6pgaLSL17FCh68CCCWrQge246k5ImsffYXj7a85G/wzLGGK84mSC+BDqLSKKIhACpwNLKBUSklbjfIBORvu54Dqvqn1Q1QVU7uOt9oKoTHYzVERIcTMy4cRz79FMu1I7ER8XbI6/GmHrDsQShqiXANGAlrieR5qvqJhG5VURudRe7BtgoIhuAfwCp2sDek48ZPw4CAsh9cwFpXdNYe2At3x7+1t9hGWPMGdl8EGdB5m2/5viXX9LqX2/zs6UjuKz9ZTx48YP+DssYY2w+CH+LTUulNCcH/fDfXH3u1by7812+z//e32EZY8xpWYI4CyL69yekfXuy35jLdUnXUVxWzPwt8/0dljHGnJYliLNAAgKISUslf/16Wu7NZ2DCQOZtmUdRaZG/QzPGmFOyBHGWxIwahYSGukZ5TZrIDwU/8O7Od/0dljHGnJIliLMkMCaGJsOHc+Ttt7kgugfnxpzL6xmv2+Qmxpg6yxLEWRR7bRp6/Di5S5dyXdJ1bP5hM2sO1L2nrowxBixBnFXhPXsS1r07OenpjEgcQUxojI3yaoypsyxBnGWxaakUbtuOrt/IuC7j+GD3B2QezfR3WMYYcxJLEGdZkxEjCIiOJntuOhPOm0CgBDJ381x/h2WMMSexBHGWBYSH03T0KHL/9S+a5Qfysw4/Y9G2RRwrPubv0Iwx5gSWIPwgNjUViovJWbCQSUmTyCvOY8n2Jf4OyxhjTmAJwg9CO3Ykon9/sufPo8c53egV14s3Mt6gTE87N5IxxpxVliD8JDY1lZK9+8hbvZpJSZPYfXQ3n2R+4u+wjDGmgiUIP4keMpiguDiy585lSPshtIxoyWsZr/k7LGOMqVDtBCEiASLSxIlgGpOKyYQ++RTde4DUrql8se8LtmVv83doxhgDeJkgROQNEWkiIpHAt8AWEfm9s6E1fOWTCeXMm8c1na8hLDDMXpwzxtQZ3rYguqlqLjAKWA60AyY5FlUjEdyqFdGDB5GzYCFNAiK4stOVLNuxjOyCbH+HZkyDNnnFZCavmOzvMHzCyXPxNkEEi0gwrgTxlqoWA2ccZU5EhonIFhHZLiJ3ePj+UhE5IiLr3Z973NvbisiHIpIhIptEZHp1Tqo+iUlNpTQ7m6MrV3Jd0nUUlhayYOsCf4dljDFeJ4jngF1AJLBaRNoDuaerICKBwLPAFUA3IE1Eunko+omq9nZ/7ndvKwF+q6pJQH9g6inq1nuRAwYQ3L4d2XPT6RTTiQvbXEj65nSKS4v9HZoxJ2lIv3mbM/MqQajqP1Q1XlWHq8t3wKAzVOsLbFfVHapaBKQDV3t5vH2qus69fBTIAOK9qVvfSEAAsalp5K9bR8GWLUxMmsjB/IOs+m6Vv0MzxjRy3t6knu6+SS0i8qKIrAMGn6FaPLCn0nomni/yA0Rkg4i8KyLdPRy7A9AH+OIUsU0RkTUisubQoUPenE6dEzO6fDKhuVwUfxEdmnTg9W9trghjjH9528V0k/sm9eVAHDAZeOQMdcTDtqpXvHVAe1VNBp4GThhvQkSigIXADPfxT96h6vOqmqKqKXFxcWc+kzooMCaGJldcQe7St9Fjx7ku6To2Ht7IhkMb/B2aMaYR8zZBlF/shwOzVXUDnhNAZZlA20rrCcDeygVUNVdV89zLy3HdDG8O4L4pvhCYo6qLvIyz3oq9No2y48c5snQpV3W6iujgaF7PeN3fYRljGjFvE8RaEVmFK0GsFJFo4EwDB30JdBaRRBEJAVKBpZULiEgrERH3cl93PIfd214EMlT1796fTv0V1rMnYd26kTM3nfCgcMZ2Gct7373H/mP7/R2aMaaR8jZB/By4A7hAVY8DIbi6mU5JVUuAacBKXDeZ56vqJhG5VURudRe7BtgoIhuAfwCp6up4vwjXexaDKz0CO7y6J1efiAgxaakUbttG/rp1pHVNQ1GbK8IY4zdB3hRS1TIRSQCudf/C/7Gqvu1FveW4XqyrvG1mpeVngGc81PuUM3dhNThNR4zg4GOPk/3GXOKf+BtD2g1hwdYF3NLrFiKCI/wdnjGmkfH2KaZHgOm4htn4Fvi1iDzsZGCNUUBEBE1HjSJ31SpKDh9mYtJEcotyWbZjmb9DM8Y0Qt52MQ0HfqaqL6nqS8AwYIRzYTVesakTXJMJLVxEnxZ96NasG3My5tgjr8aYs646o7nGVFpu6utAjEtop05E9OtHTno6lJUxMWkiO47s4D97/+NVfXvTte6xvxNTX3mbIB4GvhKRl0XkFWAt8JBzYTVusWmpFO/dS94nnzC0w1Cahze3uSKMMWedt0NtzMU1JtIi92eAqqY7GVhjFj1kCIFxzcmZm05IYAgTzpvAp1mfsuPIDn+HZoxpRE6bIETkJ+UfoDWul9/2AG3c24wDJDiY2HHjyFu9mqLMTMZ1GUdwQDBvZLzh79CMMY3ImR5zfeI03ylnHo/J1FDMuHF8P/M5cubNp8Vvf8OIjiNY+r+l3NbnNpqG2i0gY4zzTtuCUNVBp/lYcnBQcOvWRA0aRM6CBZQVFTExaSL5Jfks2tbgRx0xxtQR3r4HMcbDZ4iItHA6wMYsNi3NPZnQKs475zwuaHUBczfPpaSsxN+hGWMageoMtTELuM79eQH4DfCZiNjUow6JvHAAwe3akZ3ueh5gYtJE9h3bxwe7P/BzZMaYxsDbBFEGJKnqWFUdi2uGuEKgH/BHp4Jr7CQggNgJE8hfu5aCLVu5JOESEqISbJRXY8xZ4W2C6KCqByqtHwS6qOoPgM2N6aCmY0YjISFkp88lMCCQa5Ou5auDX7Hp+03+Ds1R9nKZMf7nbYL4RESWicgNInIDrmG7V4tIJJDjXHgmKDbWNZnQW0spzTvG6HNHExkcaa0IY4zjvE0QU4HZQG9c03++AkxV1WOqeqa5qU0txaalUnb8OLlvLyUqJIrR545mxa4VHDpeP6dYNcbUD96+Sa3Ap8AHwHvAarXR486asORkQrslkT03HVXl2q7XUlpWyrwt8/wdmjGmAfP2MdfxwH9xTfAzHvhCRK5xMjDzIxEhNjWVwq1byf/qK9o2acslbS9h/pb5FJYW+js8Y0wD5W0X0124ZpO7QVWvB/oCf3YuLFNV05EjCYiKIvsN1wxzE5Mmkl2YzfIdy89Q0xhjasbbBBGgqgcrrR/2pq6IDBORLSKyXUTu8PD9pSJypNK0ovd4W7exKZ9M6OjKlZT88AN9W/Wlc2xnXs943eaKMMY4wtsEsUJEVorIjSJyI/AOVaYSrUpEAoFngStwvTeRJiLdPBT9RFV7uz/3V7NuoxKbOgEtLiZn4UJEhElJk9iavZUv93/p79CMMQ2Qtzepfw88D/QCkoHnVfVML8j1Bbar6g5VLQLSgau9jKs2dRus0HPPJaJvX3LS56GlpQzvOJzY0FibK8IY4wivZ5RT1YWq+htVvV1VF3tRJR7X0ODlMt3bqhogIhtE5F0R6V7NuojIFBFZIyJrDh1q+I99xqalUpyVxbFPPyU0MJRx543j4z0fsyd3z5krG2NMNZxpPoijIpLr4XNURHLPsG/xsK1qZ/k6oL2qJgNPA0uqUde1UfV5VU1R1ZS4uLgzhFT/RQ8ZQmDz5mTPdY3PNOG8CQQGBPLGZpsrwhjjW2ca7jtaVZt4+ESrapMz7DsTaFtpPQHYW2X/uaqa515eDgSLSHNv6jZWEhJCzLhryPv4Y4oys2gR0YKhHYayePti8ory/B2eMaYB8bqLqQa+BDqLSKKIhACpuIboqCAirURE3Mt93fEc9qZuYxY7fjyIkDN/PgCTkiZxrPgYS7YvOUNNY4zxnmMJQlVLgGnASiADmK+qm0TkVhG51V3sGmCjiGwA/gGkqovHuk7FWt8Et25N1KWXkrNwIWVFRXRv3p0+LfowJ2OOPfJqjPEZJ1sQqOpyVe2iqp1U9UH3tpmqOtO9/IyqdlfVZFXtr6r/Pl1d86PYtDRKDx/m6Kp/AXBd0nVk5mVypPCInyMzxjQUjiYI45zIiy50TybkerN6SLshtIpsxYHjB85Q0xhjvGMJop5yTSY0nvw1aynYupWggCCu7XotR4uPsjdvL/kl+f4O0RhTz1mCqMeajhmDhISQ456SdPx544kJjWHvsb0MXzSceZvnUVxm8zkZY2rGEkQ95ppMaBhH3lpK2bFjRAZHcvdrhUx9P4R20e346xd/5eolV/POjnco0zJ/h2uMqWcsQdRzMamplB07xpG3l1VsSzwcyMvDXubZIc8SHhTOHZ/cwfi3x7M6c7U95WSM8ZoliHouvHdvQpOSyJ4794SLv4gwMGEgb175Jo/+9FGOFR9j6vtTuXHFjaw7sM6PERtj6gtLEPVcxWRCW7aQ/9X6k74PkACGdxzO0lFLubvf3ew+upsbVtzAtPenseWHLX6I2BhTX1iCaACajhxBQGRkxSOvngQHBuEOgIYAABisSURBVDOh6wTeGf0O038ynXUH1zHu7XHc8ckd7DlqA/0ZY05mCaIBCIiMdE0m9O4KAkpPfzM6IjiCm3vezLtj3uWmHjfx/nfvc9Xiq/jr53/l0PGGPxquMcZ7liAaiPLJhCJzvXustWloU2acP4PlY5YztstYFm5dyIjFI3hq3VPkFp1poF5jTGNgCaKBCO3cmYiUFKJzi6EaTyrFRcRxd/+7eWvUW1za9lJmfTOLKxZewUsbX7KX7Yxp5CxBNCCx16YRVKLEfl9IWWFhteq2a9KOxwY+xptXvklyXDL/t/b/GLFoBPO3zLeX7YxppCxBNCDRw4aR2zSY6Nxivku7lqLdu6u9j67ndOWfl/2Tl4e9TEJ0Ag98/gCjlozi3Z3v2st2xjQyliAaEAkIIKd5GIdahVOUmcnOMWPJXbWqRvs6v+X5vDLsFZ4Z/AyhQaH8YfUfmLBsAp9mfWov2xnTSFiCaIDyI4NIXLSIkMREsn49nf0PPYQWFVV7PyLCJW0vYcGVC3j4pw9ztOgov3zvl0xeOZn1B09+58IY07BYgmigQhLi6TDndWInTSL71dfYNXESxVlZNdpXgAQwsuNI3h71Nnf1u4tdR3Yx6d1J3PbBbWzL3ubjyI0xdUWQvwMwzpGQEFrddScR55/PvrvvZseYsbR55GGiBw2q0f6CA4NJ7ZrKVZ2uYk7GHGZvnM3YpWMZ2XEkv+r9KxKiE3x8BsZ4r7ismIKSAgpLC8kvyaewpJCC0oKKbQUlBRXrB44fQFV58ZsXUVxdpqqKoj/+RHH9v8r205WrtK28K9ZTnROOV4Oy5ccso4z/5fyPwIBAR/5MHU0QIjIMeAoIBGap6iOnKHcB8DkwQVUXuLfdDtwMKPANMFlVC5yMt6FqMmwoYUldyZxxO5m//BXn/PwmWsyYgQQH12h/EcER/KLXLxh/3nhe3Pgib2S8wbu73mVcl3FM6TWF5uHNfXwGpj5SVUrKSsgvdV+sK12gT7hwV1rOL8n/8WLu4cJ+UpnSH8uUamm1Y3xy3ZPVKi8IIkIAASDudfe28p+Vy5X/r6JspW21LRsgrg6g/JJ8ggKcuZQ7liBEJBB4FvgZkAl8KSJLVfVbD+UexTX/dPm2eODXQDdVzReR+UAq8LJT8TZ0Ie3b0yF9LgceepgfXnyJ/K/WE//3Jwhu1arG+2wa2pTfnP8brut6Hc99/Rzzt8xnyfYlTEyayOQek4kOifbhGRh/OF58nKy8LDKPZpKZl8nuo7spLSvl9x///pS/nVe+iNf0ybewwDBCg0IJCwwjLCisYj08MJyoiChCA0MJDwonNDC04vuwoDDXevmyu3zl/ZTX+93HvyOAAJ67/LlTX6A9XPjroskrJju2bydbEH2B7aq6A0BE0oGrgW+rlLsNWAhc4CG2cBEpBiKAvQ7G2igEhIbS+r57ibjgAvbdcw87R4+hzWOPEfXTi2u135aRLblnwD3c0P0Gnv3qWV745gXmb53PzT1uJrVrKmFBYT46A+NrpWWlHMo/xJ6jeyqSQPnPrKNZHC44fEL5AAkgSILY/MPmHy/IQWE0CWlywgU4NDD05It41Qt5lfXK9Zy+IAcHuFrP4UHhjh6nvnMyQcQDlUeBywT6VS7gbimMBgZTKUGoapaI/A3YDeQDq1TV4/OaIjIFmALQrl07X8ZfL6Xf1h2Aoacp03TkCMK6JZE1fQZ7pkyh2a23EDdtGhJYu37M9k3a89gljzG5x2Se+uopnlj7BK9lvMYvk3/JqHNHOdYMNqeXV5R3Qitgz9E9FQkgKy/rhBchAySA1pGtSYhK4NK2l5IQnUBCVELFzxkfzkBEmD1sth/PyJwtTv4X6+lXgKoP0D8J/FFVSyv/xiAisbhaG4lADvCmiExU1ddP2qHq88DzACkpKfaAvpdCO3akw/x57H/grxz+fzPJX7uONn97nOAWLWq976RmScy8bCZf7v+Sp9Y9xX3/uY9XNr3CtD7T+Fn7n1X0nRrfKCkr4cDxA64EULkVcDSTrLwssguzTyjfJKQJCdEJdIntwuB2g0mITiA+Kp62UW1pFdWq4rdrT+pyV4vxPScTRCbQttJ6Aid3E6UA6e5/dM2B4SJSAgQDO1X1EICILAIuBE5KEKbmAsLDafPQg0SkpLD//vvZOWYs8X97nMj+/X2y/wtaXcBrV7zGR3s+4h9f/YPfffw7ks5JYsZPZjCgzQC72FTDkcIjJ7QCKieDfXn7KNGSirJBEkTrKFcr4LL2l53QCoiPiqdpaFM/nompT5xMEF8CnUUkEcjCdZP52soFVDWxfFlEXgaWqeoSEekH9BeRCFxdTEOANQ7G2qjFjBlNWI/uZM24nd03/Zzm06bS/NZbkYDa/6YvIgxqN4iBCQNZvnM5z65/llveu4W+rfoy/SfT6RXXywdnUP8VlxWzP28/e/L2eEwCR4uOnlA+NjSWhOgEejTrwbAOw05IAi0iWlh3nvEJx/4VqWqJiEzD9XRSIPCSqm4SkVvd3888Td0vRGQBsA4oAb7C3Y1knBHWpQuJb85n37338f0/nnZ1OT3+GEHnnOOT/QcGBHJlpysZ2mEoC7Yu4Lmvn+O65dcxuO1gbutzG+fGnuuT49RVqkp2QXZFt89JrYBj+0544ic4IJj4qHgSohNIjks+qRUQFRLlx7MxjYWjv2ao6nJgeZVtHhODqt5YZf0vwF8cC86cJCAykjaPPUrEBSkc+OuD7Bw1mvi/P0FESorPjhESGMK1Sdcy6txRvPbta7y86WXGLB3DlZ2u5Fe9f0V8VLzPjuVLZVrG8eLj5BXnkVeU5/pZ/inK41jxsRO/c/88VnyMbdnbKCorYuC8gSfss1lYMxKiE+jdojcjo0ae1AqwezXG36wdak4gIsSOH094z55kzpjBdzfcSNyM6TT7+c990uVULiI4gluSb2HCeRN4ceOLzN08l+U7lzPhvAn8oucvfHYcVSW/JP/kC3ilC/vR4qMcKzp2wvbyi/vRoqMcKz7GseJjFW+1noogRAZHEhUSRVSw69MktAnhQeE0CWjC5B6TK5JAm6g2RARH+Ow8jXGCJYgGxlePH4YlJZG4cCH77v4zh574O/lr1tL6kYcJio31yf7LxYTF8NuU33Jd0nXM3DCT9M3pLNq2iJjQGFpEtOCHgh88/lZefuEuv7gfLXateyrjzRu24UHhrou6++IeGRxJi4gWrgt+pe1RwVFEhkQSHRxNZHAk0SHRFWUigiM8/tZf/iLTxG4TffpnZ4zTLEGYUwqMiiL+//5OdkoKBx59lJ1jxpLwf38nvHdvnx+rVWQr7r3wXm7sfiPPrH+GlbtWsu/YPi6Zd8lp64UEhJxwUY8OiSY+Kv6EC3fli3vlspWX7aauMSez/yrMaYkI50y8jvDkXmTNuJ1dEyfR4ne/5ZwbbnDkMdUOTTvwt0v+xu7c3eQW5XJD9xtOuLhHBrt/ew9xXfxDAkN8HoMxxsUShPFKeM+eJC5ayN477+LgI4+Sv3YtrR98kMAmTRw5XmRwJJHBkaR1TXNk/8aYM7PHJIzXAps2JeGZp2nxxz9y9MOP2DlmLPkbN/k7LGOMQyxBmGoREZpNvpH2r72KlpTwXVoaP7zxhk1DakwDZAnC1EhEnz4kLl5ExID+HLj/Afb+9reU5uX5OyxjjA9ZgjA1FhQbS9uZM4n7zW/IXbGSXWOvoWDzZn+HZYzxEUsQplYkIIDmU35B+1depuz4cXZNSCX7zTety8mYBsAShPGJiAsuIHHJYiLO/wn7/3wP++64g7Ljx/0dljGmFixBGJ8JataMti+8QPPbpnFk6dvsHDeewu3b/R2WMaaGLEEYn5LAQOKmTqXdSy9SmpPDznHjyVmyxN9hGWNqwBKEcUTkgAEkLl5EeI8e7LvjT+y9+27KCgr8HZYxphosQRjHBLdoQbvZL9Hslls4smAhuyakUrhzp1d1U5/eROrT9hKeMf5kCcI4SoKCaHH7DNq+8DwlBw6wa+w1HHnnHX+HZYzxgiUIc1ZE/fSnJC5eROh557H3t79j3333UVZY6O+wjDGn4WiCEJFhIrJFRLaLyB2nKXeBiJSKyDWVtsWIyAIR2SwiGSIywMlYjfOCW7em/auvcM5NN5EzN53v0q6laPduf4dljDkFxxKEiAQCzwJXAN2ANBHpdopyj+Kau7qyp4AVqtoVSAYynIrVnD0SHEzLP/yehH8+S1FmJjvHjCV31Sp/h2WM8cDJFkRfYLuq7lDVIiAduNpDuduAhcDB8g0i0gQYCLwIoKpFqprjYKzmLIsePJjERYsISUwk69fT2f/QQ2hRkb/DMsZU4mSCiAf2VFrPdG+rICLxwGhgZpW6HYFDwGwR+UpEZolIpKeDiMgUEVkjImsOHTrku+iN40IS4ukw53ViJ00i+9XX2DVxEsVZWf4Oyxjj5mSC8DTdWNUBep4E/qh60qTBQcBPgP+nqn2AY4DHexiq+ryqpqhqSlxcXG1jNmeZhITQ6q47iX/qKYp27GDHmLEc/fBDf4dljMHZBJEJtK20ngDsrVImBUgXkV3ANcA/RWSUu26mqn7hLrcAV8IwDVSToZeTuHABwW3akPnLXxFzuABswD9j/MrJKUe/BDqLSCKQBaQC11YuoKqJ5csi8jKwTFWXuNf3iMh5qroFGAJ862Cspg4Iad+eDulzOfDww5A+j7DjpRx4+GFCuyYRltSV0I4dkZD6Nwd1xQt/w/wbhzHV5ViCUNUSEZmG6+mkQOAlVd0kIre6v69636Gq24A5IhIC7AAmOxWrqTsCQkNpfe+9bP/gLaKPFJE9bz5aPkRHcDCh555LWNeuroTRtSthXbs6Ni+2MY2dky0IVHU5sLzKNo+JQVVvrLK+HlcXlGmEjkcHczw6mMuX/oei776jICODws2bKcjYTN7q1RxZvLiibHB8PKFJXQlztzTCunYlqE0bRDzdBjO10VBaQw3lPMDZc3E0QRhTWxIYSGjHjoR27AgjRlRsLzl0iIKMDAoyNlOwOYPCjM3kvf9BxX2LgCZNKrU06ncXlTH+YgnC1EtBcXFExcURNXBgxbayY8co2Lq1oqVRsHkz2enzUPeQHhIcTEjnc10tjUrdVIHR0f46DWPqNEsQpsEIiIwkok8fIvr0qdimJSXuLqrNFG52tTjyPvqII4sWVZQJTkj48Z5Gkit5BLVubV1UptGzBGEaNAkKIrRTJ0I7dYKRri4qVaXk0KFKLQ1XF9XR997/sYuqaVNXK6NrV9f9jaQkVxdVcLA/T8eYs8oShGl0RITgFi0IbtHCiy6qdOuiMo2WJQhj3HzTRdWNsKSuBLVqZV1Upt6zBGHMadS0iyqwaVNC3fczIo4WUxok5H/9NRIaRkB4mOtnWCgSFoaEhFgyMXWSJQhjqqlaXVRz59Lc3UW1a/yEU+0QCQ0lICwMCQsjINSdOMJCCQgNQ8LDXD/L18PcyaVqsikvExZWsa+K/ZaXCQ9HgoMtIRmvWIIwdVL6bd0BGOrnOKrjVF1UH1zRj8AS5fx7nkALCykrKEALCtw/CykrdP3UwgLKCgrRgnz3zwLKCgsp+/4wxe4yFXULC6G4uGaBipyQiAJCXYnjpMTkIUFFZxeBwOHZL7v3VXm3UrH/SltPOO5J359QVE7cz6n25YP6kbmuP7ucxUuo7yJzi1GHRtWzBGGMgyQoiJKQQEpCIHrQIJ/uW0tLK5LFjwnHnWQK3es1Skh5p0xIse5jH3z0UZ+ey9nWzP1z35/+5Nc4fKEZUBroTIvQEoQx9ZQEBiKRkQREepwqxee0tJR/jewPwJA3PzxxtN3yZU/bcN23OXmHNah/wm68r1++XL5p9U2uucsGvlT/WxCrbxoFQA8H9m0JwhjjFQkMRANcv6kGRkX5OZraKQ129cmEJCT4OZLaKz8XJzg5H4Qxxph6zBKEMcYYjyxBGGOM8cgShDHGGI8sQRhjjPHI0QQhIsNEZIuIbBeRO05T7gIRKRWRa6psDxSRr0RkmZNxGmOMOZljj7mKSCDwLPAzIBP4UkSWquq3Hso9imvu6qqmAxmATTps6q36+FZ4Q9f1nK7+DsFnnDwXJ9+D6AtsV9UdACKSDlwNfFul3G3AQuCCyhtFJAEYATwI/MbBOI0xXmooya79a6/6OwSfcfJcnEwQ8cCeSuuZQL/KBUQkHhgNDKZKggCeBP4A2GD7jdDsYbP9HYLxwP5eGhcn70F4Ghyk6vv2TwJ/VNXSEyqKjAQOquraMx5EZIqIrBGRNYcOHap5tMYYY07gZAsiE2hbaT0B2FulTAqQ7h59sTkwXERKcLU0rhKR4UAY0EREXlfViVUPoqrPA88DpKSkeBjwxRhjTE04mSC+BDqLSCKQBaQC11YuoKqJ5csi8jKwTFWXAEuAP7m3Xwr8zlNyMMYY4xzHEoSqlojINFxPJwUCL6nqJhG51f39TKeObUxdYv32pr4Sj8Pw1lMpKSm6Zs0af4dhjDH1hoisVdUUT9/Zm9TGGGM8sgRhjDHGI0sQxhhjPLIEYYwxxiNLEMYYYzyyBGGMMcYjSxDGGGM8sgRhjDHGowb1opyIHAK+q2H15sD3PgzHnxrKuTSU8wA7l7qooZwH1O5c2qtqnKcvGlSCqA0RWXOqtwnrm4ZyLg3lPMDOpS5qKOcBzp2LdTEZY4zxyBKEMcYYjyxB/Oh5fwfgQw3lXBrKeYCdS13UUM4DHDoXuwdhjDHGI2tBGGOM8cgShDHGGI8sQVQiIg+IyNcisl5EVolIG3/HVBMi8riIbHafy2IRifF3TDUlIuNEZJOIlIlIvXskUUSGicgWEdkuInf4O57aEJGXROSgiGz0dyy1ISJtReRDEclw/9ua7u+YakpEwkTkvyKywX0u9/l0/3YP4kci0kRVc93Lvwa6qeqtfg6r2kTkcuAD97SvjwKo6h/9HFaNiEgSUAY8h2tu8nozZaCIBAJbgZ8BmbjmaU9T1W/9GlgNichAIA94VVV7+DuemhKR1kBrVV0nItHAWmBUffx7EREBIlU1T0SCgU+B6ar6uS/2by2ISsqTg1skUC+zp6quUtUS9+rnQII/46kNVc1Q1S3+jqOG+gLbVXWHqhYB6cDVfo6pxlR1NfCDv+OoLVXdp6rr3MtHgQwg3r9R1Yy65LlXg90fn123LEFUISIPisge4DrgHn/H4wM3Ae/6O4hGKh7YU2k9k3p6IWqoRKQD0Af4wr+R1JyIBIrIeuAg8C9V9dm5NLoEISLvichGD5+rAVT1LlVtC8wBpvk32lM703m4y9wFlOA6lzrLm3Opp8TDtnrZKm2IRCQKWAjMqNJ7UK+oaqmq9sbVU9BXRHzW/Rfkqx3VF6p6mZdF3wDeAf7iYDg1dqbzEJEbgJHAEK3jN5qq8XdS32QCbSutJwB7/RSLqcTdX78QmKOqi/wdjy+oao6IfAQMA3zyIEGja0Gcjoh0rrR6FbDZX7HUhogMA/4IXKWqx/0dTyP2JdBZRBJFJARIBZb6OaZGz31j90UgQ1X/7u94akNE4sqfUhSRcOAyfHjdsqeYKhGRhcB5uJ6a+Q64VVWz/BtV9YnIdiAUOOze9Hl9fBoLQERGA08DcUAOsF5Vh/o3Ku+JyHDgSSAQeElVH/RzSDUmInOBS3ENLX0A+IuqvujXoGpARC4GPgG+wfXfOsCdqrrcf1HVjIj0Al7B9e8rAJivqvf7bP+WIIwxxnhiXUzGGGM8sgRhjDHGI0sQxhhjPLIEYYwxxiNLEMYYYzyyBGFMNYhI3plLnbb+AhHp6F6OEpHnROR/7pE4V4tIPxEJcS83uhdZTd1iCcKYs0REugOBqrrDvWkWrsHvOqtqd+BGoLl7YL/3gQl+CdQYN0sQxtSAuDzuHjPqGxGZ4N4eICL/dLcIlonIchG5xl3tOuAtd7lOQD/gblUtA3CP+vqOu+wSd3lj/MaasMbUzBigN5CM683iL0VkNXAR0AHoCbTANZT0S+46FwFz3cvdcb0VXnqK/W8ELnAkcmO8ZC0IY2rmYmCueyTNA8DHuC7oFwNvqmqZqu4HPqxUpzVwyJuduxNHkXtCG2P8whKEMTXjaSjv020HyAfC3MubgGQROd1/g6FAQQ1iM8YnLEEYUzOrgQnuyVrigIHAf3FN+TjWfS+iJa7B7cplAOcCqOr/gDXAfe7RRRGRzuVzYIhIM+CQqhafrRMypipLEMbUzGLga2AD8AHwB3eX0kJc80BsxDWP9hfAEXeddzgxYdwMtAK2i8g3wAv8OF/EIKDejS5qGhYbzdUYHxORKPck8s1wtSouUtX97vH6P3Svn+rmdPk+FgF/qsfzcZsGwJ5iMsb3lrkncQkBHnC3LFDVfBH5C655qXefqrJ7cqEllhyMv1kLwhhjjEd2D8IYY4xHliCMMcZ4ZAnCGGOMR5YgjDHGeGQJwhhjjEf/H0ZPxn/paAldAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.保存模型，用于后续测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(grid.best_estimator_, open(\"Pima-indians-diabetes_L1_org_tfidf.pkl\", 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
